import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;


public class LibSVMDataCreation {

	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		//Read dictionary
		ArrayList<String> dictionary = new ArrayList<String>();
		String fileDicName = "dictionary.txt";
		try {
			File fileDir = new File(fileDicName);

			BufferedReader in = new BufferedReader(new InputStreamReader(
					new FileInputStream(fileDir), "UTF8"));

			String str;

			while ((str = in.readLine()) != null) {
				dictionary.add(str);
			}

			in.close();
		} catch (UnsupportedEncodingException e) {
			System.out.println(e.getMessage());
		} catch (IOException e) {
			System.out.println(e.getMessage());
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
		java.util.Collections.sort(dictionary);

		//---------------------------------------------
		
			String fileVectorName = "data_doisong_temp.txt";
			String[] vectorArray = readFile(fileVectorName).split("\n");
			
			//read and save temp vector
//			List<Article> articleList = new LinkedList<Article>();
//			for(int i = 0; i < vectorArray.length; i++){
////						System.out.println(vectorArray[i]);
//				String[] tempVectorContent = null;
//				tempVectorContent = vectorArray[i].split(" ");		
//				Article tempArticle = new Article(tempVectorContent[0],"",Integer.parseInt(tempVectorContent[1]),0,0,0);
//				articleList.add(tempArticle);
//			}
			HashMap<String, Integer> articleList = new HashMap<String, Integer>();
			for(int i = 0; i < vectorArray.length; i++){
				String[] tempVectorContent = null;
				tempVectorContent = vectorArray[i].split(" ");		
				articleList.put(tempVectorContent[0], Integer.parseInt(tempVectorContent[1]));
			}
			
			File f = new File("data_doisong.txt");
		    FileWriter fw = new FileWriter(f, true);
		    BufferedWriter bw = new BufferedWriter(fw);		
		    String line;
			
			//read document
			String fileWithoutPuctuation= "";
			int numOfArticle = 1100;
			
			List<Article> articleVector;
			
			for(int i = 0; i < numOfArticle; i++){
				
				String fileName = "";
				fileName = "doisong (" + (i+1) + ").txt";
//				if((i + 1) < 10){
//					fileName = "vnexpress000" + (i+1) + ".doisong";
//				} else if((i + 1)/10 >= 1 && (i + 1)/10 < 10){
//					fileName = "vnexpress00" + (i+1) + ".doisong";
//				} else if((i + 1)/100 >= 1 && (i + 1)/100 < 10){
//					fileName = "vnexpress0" + (i+1) + ".doisong";
//				} else if((i + 1)/1000 >= 1 && (i + 1)/1000 < 10){
//					fileName = "vnexpress" + (i+1) + ".doisong";
//				}
				
				fileWithoutPuctuation = readFile(fileName);
				//remove special characters
				fileWithoutPuctuation = fileWithoutPuctuation.replaceAll("[^\\p{L}\\p{Nd}]"," ");
				//---------------------------------------
				String[] articleWords = fileWithoutPuctuation.split(" ");
				
				LinkedHashMap <String, Integer> linkedHashMap = new LinkedHashMap <String, Integer>();
				
				//save word hashmap which contains unique words of document and its frequency 
				for(int j = 0; j < articleWords.length; j++){
					Integer freq = linkedHashMap.get(articleWords[j]);
					if(freq == null){
						freq = 1;
					} else {
						freq++;
					}					
					if(dictionary.contains(articleWords[j])){
						linkedHashMap.put(articleWords[j], freq);
					} 				
				}

				List<Article> articleDocList = new LinkedList<Article>();
				for (Entry<String, Integer> entry : linkedHashMap.entrySet()) {
					String key = entry.getKey();
					Integer value = entry.getValue();
					Article article = new Article();
					article.setWord(key);
					article.setNumOfWordADoc(value);
					articleDocList.add(article);
		        }
				
				//calculate length of document after standardize
				int docSize = 0;
				for(int k = 0; k < articleDocList.size(); k++){
					if(articleList.containsKey(articleDocList.get(k).getWord())){
						docSize += articleDocList.get(k).getNumOfWordADoc();
					}
				}
				
				
//				List<Integer> list = new ArrayList<>(hashMap.values());
//				for(int k = 0; k < list.size(); k++){
//					System.out.println(list.get(k));
//				}
				
//				System.out.println(docSize);
				int numOfWordADoc;
				int numOfWord = 0;
				articleVector = new LinkedList<Article>();
				for(int k = 0; k < articleDocList.size(); k++){
					numOfWordADoc = articleDocList.get(k).getNumOfWordADoc();
//					System.out.println("articleDocList.get(k).getWord() : " + articleDocList.get(k).getWord());
					if(articleList.containsKey(articleDocList.get(k).getWord())){
						numOfWord = articleList.get(articleDocList.get(k).getWord()); //number of documents which contains this word
					} 
					
//					if(hashMap.containsKey(word)){
						//if this word exist in this document, do: 
						if(numOfWordADoc > 2 && numOfWordADoc < 5){
							double tf = 0;
							double idf = 0;
							double tfidf = 0;
							double decimal = Math.pow(10, 8);
//							Article tempArticle = new Article();
//							tempArticle.setWord(word);
//							tempArticle.setNumOfWord(articleList.get(k).getNumOfWord());
//							tempArticle.setNumOfWordADoc(hashMap.get(word));
//							tempArticle.setDocLength(articleList.size());
							if(numOfWord > 0){
								tf 	= (double)numOfWordADoc/docSize; 
								idf = (double)Math.log10(numOfArticle/numOfWord);
								tfidf = Math.round(tf*idf*decimal)/decimal;
								if(tfidf > 0) {
									articleDocList.get(k).setTfidf(BigDecimal.valueOf(tfidf).toPlainString());
								}
								
//								tempArticle.setIfidf(tfidf);
//								System.out.println("tf tuso  :  " + tempArticle.getNumOfWordADoc());
//								System.out.println("docSize  :  " + docSize);
//								System.out.println("tf  :  " + tf);
//								System.out.println("idf :  " + idf);
//								System.out.printf(tempArticle.getTfidf());
//								System.out.println();
							} 		
						}
				}
				
//				System.out.println(hashMap);
				line = "" + 1 + " ";
				for(int k = 0; k < articleDocList.size(); k++){
					System.out.println(articleDocList.get(k).getWord());
					System.out.println(articleDocList.get(k).getNumOfWord());
					System.out.println(articleDocList.get(k).getNumOfWordADoc());
					System.out.println(articleDocList.get(k).getDocLength());
					System.out.println(articleDocList.get(k).getIfidf());
					System.out.println("-------------------------------");
					
					if(articleDocList.get(k).getTfidf() != ""){
						line += (k+1) + ":" + articleDocList.get(k).getTfidf() + " ";
					}
				}
//				System.out.println((i+1) + " " + articleList.size() + " " + articleVector.size() + " " + hashMap.size() + " " + docSize + " " + line);
				bw.write(line);
			    bw.newLine();
			    
			    linkedHashMap.clear();
			    articleVector.clear();
			    System.out.println("i : " + i);
			}		
			bw.close();
			
			

			
//			for(int i = 0; i < articleList.size(); i++){
//				System.out.println(articleList.get(i).getWord());
//				System.out.println(articleList.get(i).getNumOfWord());
//				System.out.println(articleList.get(i).getNumOfWordADoc());
//				System.out.println(articleList.get(i).getDocLength());
//				System.out.println(articleList.get(i).getIfidf());
//				System.out.println("-------------------------------");
//			}

		}
		
		static String readFile(String fileName) {
			String temp = "";
			try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
				String sCurrentLine;
				while ((sCurrentLine = br.readLine()) != null) {
					temp += sCurrentLine + "\n";
				}
			} catch (IOException e) {
				e.printStackTrace();
			}

			return temp;
		}

		static void writeFile(String fileName, String writeString) {
			try {
				File fileDir = new File(fileName);
				BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileDir), "UTF8"));
				
				out.append(writeString);
				
				out.flush();
				out.close();
			} catch (UnsupportedEncodingException e) {
				System.out.println(e.getMessage());
			} catch (IOException e) {
				System.out.println(e.getMessage());
			} catch (Exception e) {
				System.out.println(e.getMessage());
			}
		}
	}
